{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "620d3e22-b639-4203-853d-8ba65247a425",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.float_format', lambda x: '{:.2f}'.format(x))\n",
    "\n",
    "import seaborn as sns\n",
    "color = sns.color_palette()\n",
    "sns.set_style('darkgrid')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from scipy import stats\n",
    "from scipy.special import boxcox1p\n",
    "from scipy.stats import norm, skew\n",
    "\n",
    "#忽略警告\n",
    "import warnings\n",
    "def ignore_warn(*args, **kwargs):\n",
    "    pass\n",
    "warnings.warn = ignore_warn\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "fea633fc-867f-466f-8220-ab6de612763e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of training data: (1460, 80)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotArea Street Alley LotShape LandContour  \\\n",
       "0   1          60       RL     8450   Pave   NaN      Reg         Lvl   \n",
       "1   2          20       RL     9600   Pave   NaN      Reg         Lvl   \n",
       "2   3          60       RL    11250   Pave   NaN      IR1         Lvl   \n",
       "3   4          70       RL     9550   Pave   NaN      IR1         Lvl   \n",
       "4   5          60       RL    14260   Pave   NaN      IR1         Lvl   \n",
       "\n",
       "  Utilities LotConfig  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0    AllPub    Inside  ...        0    NaN   NaN         NaN       0      2   \n",
       "1    AllPub       FR2  ...        0    NaN   NaN         NaN       0      5   \n",
       "2    AllPub    Inside  ...        0    NaN   NaN         NaN       0      9   \n",
       "3    AllPub    Corner  ...        0    NaN   NaN         NaN       0      2   \n",
       "4    AllPub       FR2  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "   YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0    2008        WD         Normal     208500  \n",
       "1    2007        WD         Normal     181500  \n",
       "2    2008        WD         Normal     223500  \n",
       "3    2006        WD        Abnorml     140000  \n",
       "4    2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')\n",
    "train.drop('LotFrontage', axis=1, inplace=True)\n",
    "test.drop('LotFrontage', axis=1, inplace=True)\n",
    "print('The shape of training data:', train.shape)\n",
    "train.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2d9665da-afcc-4ecf-8610-ac200e4c864e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#ID列没有用，直接删掉\n",
    "# train.drop('Id', axis=1, inplace=True)\n",
    "# test.drop('Id', axis=1, inplace=True)\n",
    "\n",
    "# print('The shape of training data:', train.shape)\n",
    "# print('The shape of testing data:', test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5ee5cf24-1ac6-46eb-be9b-ab356b2a105e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     1460.00\n",
       "mean    180921.20\n",
       "std      79442.50\n",
       "min      34900.00\n",
       "25%     129975.00\n",
       "50%     163000.00\n",
       "75%     214000.00\n",
       "max     755000.00\n",
       "Name: SalePrice, dtype: float64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['SalePrice'].describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "6bdbfe68-f09d-4d6c-ac23-da6993357354",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of numeric features: 36\n",
      "number of categorical features: 43\n",
      "['Id', 'MSSubClass', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'MoSold', 'YrSold']\n"
     ]
    }
   ],
   "source": [
    "#分离数字特征和类别特征\n",
    "num_features = []\n",
    "cate_features = []\n",
    "for col in test.columns:\n",
    "    if test[col].dtype == 'object':\n",
    "        cate_features.append(col)\n",
    "    else:\n",
    "        num_features.append(col)\n",
    "print('number of numeric features:', len(num_features))\n",
    "print('number of categorical features:', len(cate_features))\n",
    "print(num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "21626804-9630-485f-87f0-59d0201bd6ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of training data: (1460, 80)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PoolQC          1453\n",
       "MiscFeature     1406\n",
       "Alley           1369\n",
       "Fence           1179\n",
       "MasVnrType       872\n",
       "FireplaceQu      690\n",
       "GarageType        81\n",
       "GarageYrBlt       81\n",
       "GarageFinish      81\n",
       "GarageQual        81\n",
       "GarageCond        81\n",
       "BsmtExposure      38\n",
       "BsmtFinType2      38\n",
       "BsmtQual          37\n",
       "BsmtCond          37\n",
       "BsmtFinType1      37\n",
       "MasVnrArea         8\n",
       "Electrical         1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看训练集中各特征的数据缺失个数\n",
    "print('The shape of training data:', train.shape)\n",
    "train_missing = train.isnull().sum()\n",
    "train_missing = train_missing.drop(train_missing[train_missing==0].index).sort_values(ascending=False)\n",
    "train_missing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ac45c610-1733-4b2a-9769-743718eb7d46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of testing data: (1459, 79)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PoolQC          1456\n",
       "MiscFeature     1408\n",
       "Alley           1352\n",
       "Fence           1169\n",
       "MasVnrType       894\n",
       "FireplaceQu      730\n",
       "GarageQual        78\n",
       "GarageYrBlt       78\n",
       "GarageFinish      78\n",
       "GarageCond        78\n",
       "GarageType        76\n",
       "BsmtCond          45\n",
       "BsmtExposure      44\n",
       "BsmtQual          44\n",
       "BsmtFinType2      42\n",
       "BsmtFinType1      42\n",
       "MasVnrArea        15\n",
       "MSZoning           4\n",
       "Functional         2\n",
       "BsmtFullBath       2\n",
       "BsmtHalfBath       2\n",
       "Utilities          2\n",
       "KitchenQual        1\n",
       "TotalBsmtSF        1\n",
       "BsmtUnfSF          1\n",
       "GarageCars         1\n",
       "GarageArea         1\n",
       "BsmtFinSF2         1\n",
       "BsmtFinSF1         1\n",
       "Exterior2nd        1\n",
       "Exterior1st        1\n",
       "SaleType           1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看测试集中各特征的数据缺失个数\n",
    "print('The shape of testing data:', test.shape)\n",
    "test_missing = test.isnull().sum()\n",
    "test_missing = test_missing.drop(test_missing[test_missing==0].index).sort_values(ascending=False)\n",
    "test_missing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ffa98d00-1154-41e2-9439-ef80605b517f",
   "metadata": {},
   "outputs": [],
   "source": [
    "none_lists = ['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtFinType1',\n",
    "              'BsmtFinType2', 'BsmtCond', 'BsmtExposure', 'BsmtQual', 'MasVnrType']\n",
    "for col in none_lists:\n",
    "    train[col] = train[col].fillna('None')\n",
    "    test[col] = test[col].fillna('None')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "95a31d8a-fce5-4a70-8031-d8f8b276a70b",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_lists = ['MSZoning', 'Exterior1st', 'Exterior2nd', 'SaleType', 'KitchenQual', 'Electrical']\n",
    "for col in most_lists:\n",
    "    train[col] = train[col].fillna(train[col].mode()[0])\n",
    "    test[col] = test[col].fillna(train[col].mode()[0])    #注意这里补充的是训练集中出现最多的类别\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2c60dba2-5f35-4192-a45f-fed8aeff8b20",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['Functional'] = train['Functional'].fillna('Typ')\n",
    "test['Functional'] = test['Functional'].fillna('Typ')\n",
    "\n",
    "train.drop('Utilities', axis=1, inplace=True)\n",
    "test.drop('Utilities', axis=1, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "a7ca09b2-bedc-49aa-8c64-81e8bec8b938",
   "metadata": {},
   "outputs": [],
   "source": [
    "zero_lists = ['GarageYrBlt', 'MasVnrArea', 'BsmtFullBath', 'BsmtHalfBath', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'GarageCars', 'GarageArea',\n",
    "              'TotalBsmtSF']\n",
    "for col in zero_lists:\n",
    "    train[col] = train[col].fillna(0)\n",
    "    test[col] = test[col].fillna(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "26d5a5a0-c644-4b8e-b097-86d5c282be65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train['LotFrontage'] = train.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))\n",
    "# for ind in test['LotFrontage'][test['LotFrontage'].isnull().values==True].index:\n",
    "#     x = test['Neighborhood'].iloc[ind]\n",
    "#     test['LotFrontage'].iloc[ind] = train.groupby('Neighborhood')['LotFrontage'].median()[x]\n",
    "# 搞不定这一列直接删了\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "42fe69fd-d46e-49d3-98d5-24b025fd8b15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum().any()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "9accb734-6632-4431-a063-934c855fde0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum().any()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5d4f24c6-db23-4720-a33e-e185da29bca4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of categorical features: 42\n"
     ]
    }
   ],
   "source": [
    "#从存放类别特征的列表去掉'Utilities'\n",
    "cate_features.remove('Utilities')\n",
    "print('The number of categorical features:', len(cate_features))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "9f42f855-a79e-4fc3-bb2c-1797a195e425",
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in cate_features:\n",
    "    train[col] = train[col].astype(str)\n",
    "    test[col] = test[col].astype(str)\n",
    "le_features = ['Street', 'Alley', 'LotShape', 'LandContour', 'LandSlope', 'HouseStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'ExterQual', \n",
    "               'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'CentralAir',\n",
    "               'KitchenQual', 'Functional', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']\n",
    "for col in le_features:\n",
    "    encoder = LabelEncoder()\n",
    "    value_train = set(train[col].unique())\n",
    "    value_test = set(test[col].unique())\n",
    "    value_list = list(value_train | value_test)\n",
    "    encoder.fit(value_list)\n",
    "    train[col] = encoder.transform(train[col])\n",
    "    test[col] = encoder.transform(test[col])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "4859019c-4f95-4e11-82ea-6c4bcce531d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MiscVal         24.45\n",
       "PoolArea        14.81\n",
       "LotArea         12.20\n",
       "3SsnPorch       10.29\n",
       "LowQualFinSF     9.00\n",
       "KitchenAbvGr     4.48\n",
       "BsmtFinSF2       4.25\n",
       "ScreenPorch      4.12\n",
       "BsmtHalfBath     4.10\n",
       "EnclosedPorch    3.09\n",
       "MasVnrArea       2.67\n",
       "OpenPorchSF      2.36\n",
       "BsmtFinSF1       1.68\n",
       "WoodDeckSF       1.54\n",
       "TotalBsmtSF      1.52\n",
       "MSSubClass       1.41\n",
       "1stFlrSF         1.38\n",
       "GrLivArea        1.37\n",
       "BsmtUnfSF        0.92\n",
       "2ndFlrSF         0.81\n",
       "OverallCond      0.69\n",
       "TotRmsAbvGrd     0.68\n",
       "HalfBath         0.68\n",
       "Fireplaces       0.65\n",
       "BsmtFullBath     0.60\n",
       "dtype: float64"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skewness = train[num_features].apply(lambda x: skew(x)).sort_values(ascending=False)\n",
    "skewness = skewness[skewness>0.5]\n",
    "skew_features = skewness.index\n",
    "skewness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "27d2169c-7f25-4a6f-ac71-d6f26c34921b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>...</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>8450</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Inside</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>9600</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>FR2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>11250</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Inside</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>9550</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Corner</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>14260</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>FR2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotArea  Street  Alley  LotShape  LandContour  \\\n",
       "0   1          60       RL     8450       1      1         3            3   \n",
       "1   2          20       RL     9600       1      1         3            3   \n",
       "2   3          60       RL    11250       1      1         0            3   \n",
       "3   4          70       RL     9550       1      1         0            3   \n",
       "4   5          60       RL    14260       1      1         0            3   \n",
       "\n",
       "  LotConfig  LandSlope  ... ScreenPorch PoolArea PoolQC Fence  MiscFeature  \\\n",
       "0    Inside          0  ...           0        0      3     4         None   \n",
       "1       FR2          0  ...           0        0      3     4         None   \n",
       "2    Inside          0  ...           0        0      3     4         None   \n",
       "3    Corner          0  ...           0        0      3     4         None   \n",
       "4       FR2          0  ...           0        0      3     4         None   \n",
       "\n",
       "   MiscVal  MoSold  YrSold  SaleType SaleCondition  \n",
       "0        0       2    2008        WD        Normal  \n",
       "1        0       5    2007        WD        Normal  \n",
       "2        0       9    2008        WD        Normal  \n",
       "3        0       2    2006        WD       Abnorml  \n",
       "4        0      12    2008        WD        Normal  \n",
       "\n",
       "[5 rows x 78 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer = train['SalePrice']\n",
    "content=train.iloc[:,0:-1]\n",
    "content.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "3f586f31-7703-4409-9fa8-ef9754a2f5a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.linear_model import LinearRegression\n",
    "# linear_model = LinearRegression()\n",
    "# linear_model.fit(content, answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "9a863ee1-f1bb-43fd-8af6-e6a44e10e668",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['IsRemod'] = 1 \n",
    "train['IsRemod'].loc[train['YearBuilt']==train['YearRemodAdd']] = 0  #是否翻新(翻新：1， 未翻新：0)\n",
    "train['BltRemodDiff'] = train['YearRemodAdd'] - train['YearBuilt']  #翻新与建造的时间差（年）\n",
    "train['BsmtUnfRatio'] = 0\n",
    "train['BsmtUnfRatio'].loc[train['TotalBsmtSF']!=0] = train['BsmtUnfSF'] / train['TotalBsmtSF']  #Basement未完成占总面积的比例\n",
    "train['TotalSF'] = train['TotalBsmtSF'] + train['1stFlrSF'] + train['2ndFlrSF']  #总面积\n",
    "#对测试集做同样的处理\n",
    "test['IsRemod'] = 1 \n",
    "test['IsRemod'].loc[test['YearBuilt']==test['YearRemodAdd']] = 0  #是否翻新(翻新：1， 未翻新：0)\n",
    "test['BltRemodDiff'] = test['YearRemodAdd'] - test['YearBuilt']  #翻新与建造的时间差（年）\n",
    "test['BsmtUnfRatio'] = 0\n",
    "test['BsmtUnfRatio'].loc[test['TotalBsmtSF']!=0] = test['BsmtUnfSF'] / test['TotalBsmtSF']  #Basement未完成占总面积的比例\n",
    "test['TotalSF'] = test['TotalBsmtSF'] + test['1stFlrSF'] + test['2ndFlrSF']  #总面积\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "89dc6676-c297-4b7c-8f38-57ed7d097eed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MiscFeature',\n",
       " 'GarageType',\n",
       " 'Condition1',\n",
       " 'BldgType',\n",
       " 'LotConfig',\n",
       " 'MSZoning',\n",
       " 'MasVnrType',\n",
       " 'SaleType',\n",
       " 'Neighborhood',\n",
       " 'Electrical',\n",
       " 'Condition2',\n",
       " 'SaleCondition',\n",
       " 'RoofStyle',\n",
       " 'Heating']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_features = list(set(cate_features).difference(set(le_features)))\n",
    "dummy_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2c4195a7-9478-4949-8ac7-8a7d7027e902",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2919, 159)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data = pd.concat((train.drop('SalePrice', axis=1), test)).reset_index(drop=True)\n",
    "all_data = pd.get_dummies(all_data, drop_first=True)  #注意独热编码生成的时候要去掉一个维度，保证剩下的变量都是相互独立的\n",
    "all_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "4aaf2c2b-9d6d-4a99-a5aa-ce0f21435651",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of training data: (1460, 160)\n",
      "The shape of testing data: (1459, 159)\n"
     ]
    }
   ],
   "source": [
    "trainset = all_data[:1460]\n",
    "y = train['SalePrice']\n",
    "trainset['SalePrice'] = y.values\n",
    "testset = all_data[1460:]\n",
    "print('The shape of training data:', trainset.shape)\n",
    "print('The shape of testing data:', testset.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "3d116a47-e677-4554-9273-175f16a6eb96",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset.to_csv('train_data.csv', index=False)\n",
    "testset.to_csv('test_data.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b82e8ef9-2a5c-4247-be32-3f809c2768e9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9aa35837-46ae-430c-90e6-11caddb5f0d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a1e9e0a-7d6c-458a-b1e2-4376197b1eeb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
